# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
import json
import re
import time

import ddddocr
import execjs
import parsel
import requests
from faker import Faker
from retrying import retry
from scrapy import signals
# useful for handling different item types with a single interface
from scrapy.exceptions import IgnoreRequest

from components.middlewares.downloadmiddlewares.private.rs_6_middlewares import Rs6RetryMiddleware
from components.middlewares.downloadmiddlewares.public.company_ip_by_api import CompanyIpByApiMiddleware
from utils.tools import parse_url_params, unquote_url, urlencode


class HunanEncrypt:
    ds_original_js_code = """
    var CryptoJS = CryptoJS || (function (Math, undefined) {
        var C = {};
        var C_lib = C.lib = {};
        var Base = C_lib.Base = (function () {
            function F() {
            };
            return {
                extend: function (overrides) {
                    F.prototype = this;
                    var subtype = new F();
                    if (overrides) {
                        subtype.mixIn(overrides);
                    }
                    if (!subtype.hasOwnProperty('init') || this.init === subtype.init) {
                        subtype.init = function () {
                            subtype.$super.init.apply(this, arguments);
                        };
                    }
                    subtype.init.prototype = subtype;
                    subtype.$super = this;
                    return subtype;
                }, create: function () {
                    var instance = this.extend();
                    instance.init.apply(instance, arguments);
                    return instance;
                }, init: function () {
                }, mixIn: function (properties) {
                    for (var propertyName in properties) {
                        if (properties.hasOwnProperty(propertyName)) {
                            this[propertyName] = properties[propertyName];
                        }
                    }
                    if (properties.hasOwnProperty('toString')) {
                        this.toString = properties.toString;
                    }
                }, clone: function () {
                    return this.init.prototype.extend(this);
                }
            };
        }());
        var WordArray = C_lib.WordArray = Base.extend({
            init: function (words, sigBytes) {
                words = this.words = words || [];
                if (sigBytes != undefined) {
                    this.sigBytes = sigBytes;
                } else {
                    this.sigBytes = words.length * 4;
                }
            }, toString: function (encoder) {
                return (encoder || Hex).stringify(this);
            }, concat: function (wordArray) {
                var thisWords = this.words;
                var thatWords = wordArray.words;
                var thisSigBytes = this.sigBytes;
                var thatSigBytes = wordArray.sigBytes;
                this.clamp();
                if (thisSigBytes % 4) {
                    for (var i = 0; i < thatSigBytes; i++) {
                        var thatByte = (thatWords[i >>> 2] >>> (24 - (i % 4) * 8)) & 0xff;
                        thisWords[(thisSigBytes + i) >>> 2] |= thatByte << (24 - ((thisSigBytes + i) % 4) * 8);
                    }
                } else if (thatWords.length > 0xffff) {
                    for (var i = 0; i < thatSigBytes; i += 4) {
                        thisWords[(thisSigBytes + i) >>> 2] = thatWords[i >>> 2];
                    }
                } else {
                    thisWords.push.apply(thisWords, thatWords);
                }
                this.sigBytes += thatSigBytes;
                return this;
            }, clamp: function () {
                var words = this.words;
                var sigBytes = this.sigBytes;
                words[sigBytes >>> 2] &= 0xffffffff << (32 - (sigBytes % 4) * 8);
                words.length = Math.ceil(sigBytes / 4);
            }, clone: function () {
                var clone = Base.clone.call(this);
                clone.words = this.words.slice(0);
                return clone;
            }, random: function (nBytes) {
                var words = [];
                var r = (function (m_w) {
                    var m_w = m_w;
                    var m_z = 0x3ade68b1;
                    var mask = 0xffffffff;
                    return function () {
                        m_z = (0x9069 * (m_z & 0xFFFF) + (m_z >> 0x10)) & mask;
                        m_w = (0x4650 * (m_w & 0xFFFF) + (m_w >> 0x10)) & mask;
                        var result = ((m_z << 0x10) + m_w) & mask;
                        result /= 0x100000000;
                        result += 0.5;
                        return result * (Math.random() > .5 ? 1 : -1);
                    }
                });
                for (var i = 0, rcache; i < nBytes; i += 4) {
                    var _r = r((rcache || Math.random()) * 0x100000000);
                    rcache = _r() * 0x3ade67b7;
                    words.push((_r() * 0x100000000) | 0);
                }
                return new WordArray.init(words, nBytes);
            }
        });
        var C_enc = C.enc = {};
        var Hex = C_enc.Hex = {
            stringify: function (wordArray) {
                var words = wordArray.words;
                var sigBytes = wordArray.sigBytes;
                var hexChars = [];
                for (var i = 0; i < sigBytes; i++) {
                    var bite = (words[i >>> 2] >>> (24 - (i % 4) * 8)) & 0xff;
                    hexChars.push((bite >>> 4).toString(16));
                    hexChars.push((bite & 0x0f).toString(16));
                }
                return hexChars.join('');
            }, parse: function (hexStr) {
                var hexStrLength = hexStr.length;
                var words = [];
                for (var i = 0; i < hexStrLength; i += 2) {
                    words[i >>> 3] |= parseInt(hexStr.substr(i, 2), 16) << (24 - (i % 8) * 4);
                }
                return new WordArray.init(words, hexStrLength / 2);
            }
        };
        var Latin1 = C_enc.Latin1 = {
            stringify: function (wordArray) {
                var words = wordArray.words;
                var sigBytes = wordArray.sigBytes;
                var latin1Chars = [];
                for (var i = 0; i < sigBytes; i++) {
                    var bite = (words[i >>> 2] >>> (24 - (i % 4) * 8)) & 0xff;
                    latin1Chars.push(String.fromCharCode(bite));
                }
                return latin1Chars.join('');
            }, parse: function (latin1Str) {
                var latin1StrLength = latin1Str.length;
                var words = [];
                for (var i = 0; i < latin1StrLength; i++) {
                    words[i >>> 2] |= (latin1Str.charCodeAt(i) & 0xff) << (24 - (i % 4) * 8);
                }
                return new WordArray.init(words, latin1StrLength);
            }
        };
        var Utf8 = C_enc.Utf8 = {
            stringify: function (wordArray) {
                try {
                    return decodeURIComponent(escape(Latin1.stringify(wordArray)));
                } catch (e) {
                    throw new Error('Malformed UTF-8 data');
                }
            }, parse: function (utf8Str) {
                return Latin1.parse(unescape(encodeURIComponent(utf8Str)));
            }
        };
        var BufferedBlockAlgorithm = C_lib.BufferedBlockAlgorithm = Base.extend({
            reset: function () {
                this._data = new WordArray.init();
                this._nDataBytes = 0;
            }, _append: function (data) {
                if (typeof data == 'string') {
                    data = Utf8.parse(data);
                }
                this._data.concat(data);
                this._nDataBytes += data.sigBytes;
            }, _process: function (doFlush) {
                var data = this._data;
                var dataWords = data.words;
                var dataSigBytes = data.sigBytes;
                var blockSize = this.blockSize;
                var blockSizeBytes = blockSize * 4;
                var nBlocksReady = dataSigBytes / blockSizeBytes;
                if (doFlush) {
                    nBlocksReady = Math.ceil(nBlocksReady);
                } else {
                    nBlocksReady = Math.max((nBlocksReady | 0) - this._minBufferSize, 0);
                }
                var nWordsReady = nBlocksReady * blockSize;
                var nBytesReady = Math.min(nWordsReady * 4, dataSigBytes);
                if (nWordsReady) {
                    for (var offset = 0; offset < nWordsReady; offset += blockSize) {
                        this._doProcessBlock(dataWords, offset);
                    }
                    var processedWords = dataWords.splice(0, nWordsReady);
                    data.sigBytes -= nBytesReady;
                }
                return new WordArray.init(processedWords, nBytesReady);
            }, clone: function () {
                var clone = Base.clone.call(this);
                clone._data = this._data.clone();
                return clone;
            }, _minBufferSize: 0
        });
        var Hasher = C_lib.Hasher = BufferedBlockAlgorithm.extend({
            cfg: Base.extend(),
            init: function (cfg) {
                this.cfg = this.cfg.extend(cfg);
                this.reset();
            }, reset: function () {
                BufferedBlockAlgorithm.reset.call(this);
                this._doReset();
            }, update: function (messageUpdate) {
                this._append(messageUpdate);
                this._process();
                return this;
            }, finalize: function (messageUpdate) {
                if (messageUpdate) {
                    this._append(messageUpdate);
                }
                var hash = this._doFinalize();
                return hash;
            }, blockSize: 512 / 32,
            _createHelper: function (hasher) {
                return function (message, cfg) {
                    return new hasher.init(cfg).finalize(message);
                };
            }, _createHmacHelper: function (hasher) {
                return function (message, key) {
                    return new C_algo.HMAC.init(hasher, key).finalize(message);
                };
            }
        });
        var C_algo = C.algo = {};
        return C;
    }(Math));

    (function () {
        var C = CryptoJS;
        var C_lib = C.lib;
        var WordArray = C_lib.WordArray;
        var C_enc = C.enc;
        var Base64 = C_enc.Base64 = {
            stringify: function (wordArray) {
                var words = wordArray.words;
                var sigBytes = wordArray.sigBytes;
                var map = this._map;
                wordArray.clamp();
                var base64Chars = [];
                for (var i = 0; i < sigBytes; i += 3) {
                    var byte1 = (words[i >>> 2] >>> (24 - (i % 4) * 8)) & 0xff;
                    var byte2 = (words[(i + 1) >>> 2] >>> (24 - ((i + 1) % 4) * 8)) & 0xff;
                    var byte3 = (words[(i + 2) >>> 2] >>> (24 - ((i + 2) % 4) * 8)) & 0xff;
                    var triplet = (byte1 << 16) | (byte2 << 8) | byte3;
                    for (var j = 0;
                         (j < 4) && (i + j * 0.75 < sigBytes); j++) {
                        base64Chars.push(map.charAt((triplet >>> (6 * (3 - j))) & 0x3f));
                    }
                }
                var paddingChar = map.charAt(64);
                if (paddingChar) {
                    while (base64Chars.length % 4) {
                        base64Chars.push(paddingChar);
                    }
                }
                return base64Chars.join('');
            }, parse: function (base64Str) {
                var base64StrLength = base64Str.length;
                var map = this._map;
                var reverseMap = this._reverseMap;
                if (!reverseMap) {
                    reverseMap = this._reverseMap = [];
                    for (var j = 0; j < map.length; j++) {
                        reverseMap[map.charCodeAt(j)] = j;
                    }
                }
                var paddingChar = map.charAt(64);
                if (paddingChar) {
                    var paddingIndex = base64Str.indexOf(paddingChar);
                    if (paddingIndex !== -1) {
                        base64StrLength = paddingIndex;
                    }
                }
                return parseLoop(base64Str, base64StrLength, reverseMap);
            }, _map: 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='
        };

        function parseLoop(base64Str, base64StrLength, reverseMap) {
            var words = [];
            var nBytes = 0;
            for (var i = 0; i < base64StrLength; i++) {
                if (i % 4) {
                    var bits1 = reverseMap[base64Str.charCodeAt(i - 1)] << ((i % 4) * 2);
                    var bits2 = reverseMap[base64Str.charCodeAt(i)] >>> (6 - (i % 4) * 2);
                    words[nBytes >>> 2] |= (bits1 | bits2) << (24 - (nBytes % 4) * 8);
                    nBytes++;
                }
            }
            return WordArray.create(words, nBytes);
        }
    }());

    CryptoJS.lib.Cipher || (function (undefined) {
        var C = CryptoJS;
        var C_lib = C.lib;
        var Base = C_lib.Base;
        var WordArray = C_lib.WordArray;
        var BufferedBlockAlgorithm = C_lib.BufferedBlockAlgorithm;
        var C_enc = C.enc;
        var Utf8 = C_enc.Utf8;
        var Base64 = C_enc.Base64;
        var C_algo = C.algo;
        var EvpKDF = C_algo.EvpKDF;
        var Cipher = C_lib.Cipher = BufferedBlockAlgorithm.extend({
            cfg: Base.extend(),
            createEncryptor: function (key, cfg) {
                return this.create(this._ENC_XFORM_MODE, key, cfg);
            }, createDecryptor: function (key, cfg) {
                return this.create(this._DEC_XFORM_MODE, key, cfg);
            }, init: function (xformMode, key, cfg) {
                this.cfg = this.cfg.extend(cfg);
                this._xformMode = xformMode;
                this._key = key;
                this.reset();
            }, reset: function () {
                BufferedBlockAlgorithm.reset.call(this);
                this._doReset();
            }, process: function (dataUpdate) {
                this._append(dataUpdate);
                return this._process();
            }, finalize: function (dataUpdate) {
                if (dataUpdate) {
                    this._append(dataUpdate);
                }
                var finalProcessedData = this._doFinalize();
                return finalProcessedData;
            }, keySize: 128 / 32,
            ivSize: 128 / 32,
            _ENC_XFORM_MODE: 1,
            _DEC_XFORM_MODE: 2,
            _createHelper: (function () {
                function selectCipherStrategy(key) {
                    if (typeof key == 'string') {
                        return PasswordBasedCipher;
                    } else {
                        return SerializableCipher;
                    }
                }

                return function (cipher) {
                    return {
                        encrypt: function (message, key, cfg) {
                            return selectCipherStrategy(key).encrypt(cipher, message, key, cfg);
                        }, decrypt: function (ciphertext, key, cfg) {
                            return selectCipherStrategy(key).decrypt(cipher, ciphertext, key, cfg);
                        }
                    };
                };
            }())
        });
        var StreamCipher = C_lib.StreamCipher = Cipher.extend({
            _doFinalize: function () {
                var finalProcessedBlocks = this._process(!!'flush');
                return finalProcessedBlocks;
            }, blockSize: 1
        });
        var C_mode = C.mode = {};
        var BlockCipherMode = C_lib.BlockCipherMode = Base.extend({
            createEncryptor: function (cipher, iv) {
                return this.Encryptor.create(cipher, iv);
            }, createDecryptor: function (cipher, iv) {
                return this.Decryptor.create(cipher, iv);
            }, init: function (cipher, iv) {
                this._cipher = cipher;
                this._iv = iv;
            }
        });
        var CBC = C_mode.CBC = (function () {
            var CBC = BlockCipherMode.extend();
            CBC.Encryptor = CBC.extend({
                processBlock: function (words, offset) {
                    var cipher = this._cipher;
                    var blockSize = cipher.blockSize;
                    xorBlock.call(this, words, offset, blockSize);
                    cipher.encryptBlock(words, offset);
                    this._prevBlock = words.slice(offset, offset + blockSize);
                }
            });
            CBC.Decryptor = CBC.extend({
                processBlock: function (words, offset) {
                    var cipher = this._cipher;
                    var blockSize = cipher.blockSize;
                    var thisBlock = words.slice(offset, offset + blockSize);
                    cipher.decryptBlock(words, offset);
                    xorBlock.call(this, words, offset, blockSize);
                    this._prevBlock = thisBlock;
                }
            });

            function xorBlock(words, offset, blockSize) {
                var iv = this._iv;
                if (iv) {
                    var block = iv;
                    this._iv = undefined;
                } else {
                    var block = this._prevBlock;
                }
                for (var i = 0; i < blockSize; i++) {
                    words[offset + i] ^= block[i];
                }
            }

            return CBC;
        }());
        var C_pad = C.pad = {};
        var Pkcs7 = C_pad.Pkcs7 = {
            pad: function (data, blockSize) {
                var blockSizeBytes = blockSize * 4;
                var nPaddingBytes = blockSizeBytes - data.sigBytes % blockSizeBytes;
                var paddingWord = (nPaddingBytes << 24) | (nPaddingBytes << 16) | (nPaddingBytes << 8) | nPaddingBytes;
                var paddingWords = [];
                for (var i = 0; i < nPaddingBytes; i += 4) {
                    paddingWords.push(paddingWord);
                }
                var padding = WordArray.create(paddingWords, nPaddingBytes);
                data.concat(padding);
            }, unpad: function (data) {
                var nPaddingBytes = data.words[(data.sigBytes - 1) >>> 2] & 0xff;
                data.sigBytes -= nPaddingBytes;
            }
        };
        var BlockCipher = C_lib.BlockCipher = Cipher.extend({
            cfg: Cipher.cfg.extend({
                mode: CBC,
                padding: Pkcs7
            }),
            reset: function () {
                Cipher.reset.call(this);
                var cfg = this.cfg;
                var iv = cfg.iv;
                var mode = cfg.mode;
                if (this._xformMode == this._ENC_XFORM_MODE) {
                    var modeCreator = mode.createEncryptor;
                } else {
                    var modeCreator = mode.createDecryptor;
                    this._minBufferSize = 1;
                }
                if (this._mode && this._mode.__creator == modeCreator) {
                    this._mode.init(this, iv && iv.words);
                } else {
                    this._mode = modeCreator.call(mode, this, iv && iv.words);
                    this._mode.__creator = modeCreator;
                }
            }, _doProcessBlock: function (words, offset) {
                this._mode.processBlock(words, offset);
            }, _doFinalize: function () {
                var padding = this.cfg.padding;
                if (this._xformMode == this._ENC_XFORM_MODE) {
                    padding.pad(this._data, this.blockSize);
                    var finalProcessedBlocks = this._process(!!'flush');
                } else {
                    var finalProcessedBlocks = this._process(!!'flush');
                    padding.unpad(finalProcessedBlocks);
                }
                return finalProcessedBlocks;
            }, blockSize: 128 / 32
        });
        var CipherParams = C_lib.CipherParams = Base.extend({
            init: function (cipherParams) {
                this.mixIn(cipherParams);
            }, toString: function (formatter) {
                return (formatter || this.formatter).stringify(this);
            }
        });
        var C_format = C.format = {};
        var OpenSSLFormatter = C_format.OpenSSL = {
            stringify: function (cipherParams) {
                var ciphertext = cipherParams.ciphertext;
                var salt = cipherParams.salt;
                if (salt) {
                    var wordArray = WordArray.create([0x53616c74, 0x65645f5f]).concat(salt).concat(ciphertext);
                } else {
                    var wordArray = ciphertext;
                }
                return wordArray.toString(Base64);
            }, parse: function (openSSLStr) {
                var ciphertext = Base64.parse(openSSLStr);
                var ciphertextWords = ciphertext.words;
                if (ciphertextWords[0] == 0x53616c74 && ciphertextWords[1] == 0x65645f5f) {
                    var salt = WordArray.create(ciphertextWords.slice(2, 4));
                    ciphertextWords.splice(0, 4);
                    ciphertext.sigBytes -= 16;
                }
                return CipherParams.create({
                    ciphertext: ciphertext,
                    salt: salt
                });
            }
        };
        var SerializableCipher = C_lib.SerializableCipher = Base.extend({
            cfg: Base.extend({
                format: OpenSSLFormatter
            }),
            encrypt: function (cipher, message, key, cfg) {
                cfg = this.cfg.extend(cfg);
                var encryptor = cipher.createEncryptor(key, cfg);
                var ciphertext = encryptor.finalize(message);
                var cipherCfg = encryptor.cfg;
                return CipherParams.create({
                    ciphertext: ciphertext,
                    key: key,
                    iv: cipherCfg.iv,
                    algorithm: cipher,
                    mode: cipherCfg.mode,
                    padding: cipherCfg.padding,
                    blockSize: cipher.blockSize,
                    formatter: cfg.format
                });
            }, decrypt: function (cipher, ciphertext, key, cfg) {
                cfg = this.cfg.extend(cfg);
                ciphertext = this._parse(ciphertext, cfg.format);
                var plaintext = cipher.createDecryptor(key, cfg).finalize(ciphertext.ciphertext);
                return plaintext;
            }, _parse: function (ciphertext, format) {
                if (typeof ciphertext == 'string') {
                    return format.parse(ciphertext, this);
                } else {
                    return ciphertext;
                }
            }
        });
        var C_kdf = C.kdf = {};
        var OpenSSLKdf = C_kdf.OpenSSL = {
            execute: function (password, keySize, ivSize, salt) {
                if (!salt) {
                    salt = WordArray.random(64 / 8);
                }
                var key = EvpKDF.create({
                    keySize: keySize + ivSize
                }).compute(password, salt);
                var iv = WordArray.create(key.words.slice(keySize), ivSize * 4);
                key.sigBytes = keySize * 4;
                return CipherParams.create({
                    key: key,
                    iv: iv,
                    salt: salt
                });
            }
        };
        var PasswordBasedCipher = C_lib.PasswordBasedCipher = SerializableCipher.extend({
            cfg: SerializableCipher.cfg.extend({
                kdf: OpenSSLKdf
            }),
            encrypt: function (cipher, message, password, cfg) {
                cfg = this.cfg.extend(cfg);
                var derivedParams = cfg.kdf.execute(password, cipher.keySize, cipher.ivSize);
                cfg.iv = derivedParams.iv;
                var ciphertext = SerializableCipher.encrypt.call(this, cipher, message, derivedParams.key, cfg);
                ciphertext.mixIn(derivedParams);
                return ciphertext;
            }, decrypt: function (cipher, ciphertext, password, cfg) {
                cfg = this.cfg.extend(cfg);
                ciphertext = this._parse(ciphertext, cfg.format);
                var derivedParams = cfg.kdf.execute(password, cipher.keySize, cipher.ivSize, ciphertext.salt);
                cfg.iv = derivedParams.iv;
                var plaintext = SerializableCipher.decrypt.call(this, cipher, ciphertext, derivedParams.key, cfg);
                return plaintext;
            }
        });
    }());

    (function () {
        var C = CryptoJS;
        var C_lib = C.lib;
        var BlockCipher = C_lib.BlockCipher;
        var C_algo = C.algo;
        var SBOX = [];
        var INV_SBOX = [];
        var SUB_MIX_0 = [];
        var SUB_MIX_1 = [];
        var SUB_MIX_2 = [];
        var SUB_MIX_3 = [];
        var INV_SUB_MIX_0 = [];
        var INV_SUB_MIX_1 = [];
        var INV_SUB_MIX_2 = [];
        var INV_SUB_MIX_3 = [];
        (function () {
            var d = [];
            for (var i = 0; i < 256; i++) {
                if (i < 128) {
                    d[i] = i << 1;
                } else {
                    d[i] = (i << 1) ^ 0x11b;
                }
            }
            var x = 0;
            var xi = 0;
            for (var i = 0; i < 256; i++) {
                var sx = xi ^ (xi << 1) ^ (xi << 2) ^ (xi << 3) ^ (xi << 4);
                sx = (sx >>> 8) ^ (sx & 0xff) ^ 0x63;
                SBOX[x] = sx;
                INV_SBOX[sx] = x;
                var x2 = d[x];
                var x4 = d[x2];
                var x8 = d[x4];
                var t = (d[sx] * 0x101) ^ (sx * 0x1010100);
                SUB_MIX_0[x] = (t << 24) | (t >>> 8);
                SUB_MIX_1[x] = (t << 16) | (t >>> 16);
                SUB_MIX_2[x] = (t << 8) | (t >>> 24);
                SUB_MIX_3[x] = t;
                var t = (x8 * 0x1010101) ^ (x4 * 0x10001) ^ (x2 * 0x101) ^ (x * 0x1010100);
                INV_SUB_MIX_0[sx] = (t << 24) | (t >>> 8);
                INV_SUB_MIX_1[sx] = (t << 16) | (t >>> 16);
                INV_SUB_MIX_2[sx] = (t << 8) | (t >>> 24);
                INV_SUB_MIX_3[sx] = t;
                if (!x) {
                    x = xi = 1;
                } else {
                    x = x2 ^ d[d[d[x8 ^ x2]]];
                    xi ^= d[d[xi]];
                }
            }
        }());
        var RCON = [0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36];
        var AES = C_algo.AES = BlockCipher.extend({
            _doReset: function () {
                if (this._nRounds && this._keyPriorReset === this._key) {
                    return;
                }
                var key = this._keyPriorReset = this._key;
                var keyWords = key.words;
                var keySize = key.sigBytes / 4;
                var nRounds = this._nRounds = keySize + 6;
                var ksRows = (nRounds + 1) * 4;
                var keySchedule = this._keySchedule = [];
                for (var ksRow = 0; ksRow < ksRows; ksRow++) {
                    if (ksRow < keySize) {
                        keySchedule[ksRow] = keyWords[ksRow];
                    } else {
                        var t = keySchedule[ksRow - 1];
                        if (!(ksRow % keySize)) {
                            t = (t << 8) | (t >>> 24);
                            t = (SBOX[t >>> 24] << 24) | (SBOX[(t >>> 16) & 0xff] << 16) | (SBOX[(t >>> 8) & 0xff] << 8) | SBOX[t & 0xff];
                            t ^= RCON[(ksRow / keySize) | 0] << 24;
                        } else if (keySize > 6 && ksRow % keySize == 4) {
                            t = (SBOX[t >>> 24] << 24) | (SBOX[(t >>> 16) & 0xff] << 16) | (SBOX[(t >>> 8) & 0xff] << 8) | SBOX[t & 0xff];
                        }
                        keySchedule[ksRow] = keySchedule[ksRow - keySize] ^ t;
                    }
                }
                var invKeySchedule = this._invKeySchedule = [];
                for (var invKsRow = 0; invKsRow < ksRows; invKsRow++) {
                    var ksRow = ksRows - invKsRow;
                    if (invKsRow % 4) {
                        var t = keySchedule[ksRow];
                    } else {
                        var t = keySchedule[ksRow - 4];
                    }
                    if (invKsRow < 4 || ksRow <= 4) {
                        invKeySchedule[invKsRow] = t;
                    } else {
                        invKeySchedule[invKsRow] = INV_SUB_MIX_0[SBOX[t >>> 24]] ^ INV_SUB_MIX_1[SBOX[(t >>> 16) & 0xff]] ^ INV_SUB_MIX_2[SBOX[(t >>> 8) & 0xff]] ^ INV_SUB_MIX_3[SBOX[t & 0xff]];
                    }
                }
            }, encryptBlock: function (M, offset) {
                this._doCryptBlock(M, offset, this._keySchedule, SUB_MIX_0, SUB_MIX_1, SUB_MIX_2, SUB_MIX_3, SBOX);
            }, decryptBlock: function (M, offset) {
                var t = M[offset + 1];
                M[offset + 1] = M[offset + 3];
                M[offset + 3] = t;
                this._doCryptBlock(M, offset, this._invKeySchedule, INV_SUB_MIX_0, INV_SUB_MIX_1, INV_SUB_MIX_2, INV_SUB_MIX_3, INV_SBOX);
                var t = M[offset + 1];
                M[offset + 1] = M[offset + 3];
                M[offset + 3] = t;
            }, _doCryptBlock: function (M, offset, keySchedule, SUB_MIX_0, SUB_MIX_1, SUB_MIX_2, SUB_MIX_3, SBOX) {
                var nRounds = this._nRounds;
                var s0 = M[offset] ^ keySchedule[0];
                var s1 = M[offset + 1] ^ keySchedule[1];
                var s2 = M[offset + 2] ^ keySchedule[2];
                var s3 = M[offset + 3] ^ keySchedule[3];
                var ksRow = 4;
                for (var round = 1; round < nRounds; round++) {
                    var t0 = SUB_MIX_0[s0 >>> 24] ^ SUB_MIX_1[(s1 >>> 16) & 0xff] ^ SUB_MIX_2[(s2 >>> 8) & 0xff] ^ SUB_MIX_3[s3 & 0xff] ^ keySchedule[ksRow++];
                    var t1 = SUB_MIX_0[s1 >>> 24] ^ SUB_MIX_1[(s2 >>> 16) & 0xff] ^ SUB_MIX_2[(s3 >>> 8) & 0xff] ^ SUB_MIX_3[s0 & 0xff] ^ keySchedule[ksRow++];
                    var t2 = SUB_MIX_0[s2 >>> 24] ^ SUB_MIX_1[(s3 >>> 16) & 0xff] ^ SUB_MIX_2[(s0 >>> 8) & 0xff] ^ SUB_MIX_3[s1 & 0xff] ^ keySchedule[ksRow++];
                    var t3 = SUB_MIX_0[s3 >>> 24] ^ SUB_MIX_1[(s0 >>> 16) & 0xff] ^ SUB_MIX_2[(s1 >>> 8) & 0xff] ^ SUB_MIX_3[s2 & 0xff] ^ keySchedule[ksRow++];
                    s0 = t0;
                    s1 = t1;
                    s2 = t2;
                    s3 = t3;
                }
                var t0 = ((SBOX[s0 >>> 24] << 24) | (SBOX[(s1 >>> 16) & 0xff] << 16) | (SBOX[(s2 >>> 8) & 0xff] << 8) | SBOX[s3 & 0xff]) ^ keySchedule[ksRow++];
                var t1 = ((SBOX[s1 >>> 24] << 24) | (SBOX[(s2 >>> 16) & 0xff] << 16) | (SBOX[(s3 >>> 8) & 0xff] << 8) | SBOX[s0 & 0xff]) ^ keySchedule[ksRow++];
                var t2 = ((SBOX[s2 >>> 24] << 24) | (SBOX[(s3 >>> 16) & 0xff] << 16) | (SBOX[(s0 >>> 8) & 0xff] << 8) | SBOX[s1 & 0xff]) ^ keySchedule[ksRow++];
                var t3 = ((SBOX[s3 >>> 24] << 24) | (SBOX[(s0 >>> 16) & 0xff] << 16) | (SBOX[(s1 >>> 8) & 0xff] << 8) | SBOX[s2 & 0xff]) ^ keySchedule[ksRow++];
                M[offset] = t0;
                M[offset + 1] = t1;
                M[offset + 2] = t2;
                M[offset + 3] = t3;
            }, keySize: 256 / 32
        });
        C.AES = BlockCipher._createHelper(AES);
    }());


    function utf8_(x) {
        var f = "";
        for (var W = 0; W < x['length']; W++)
            f += String["fromCharCode"](x[W]);
        return f
    };

    function randomCoding() {
        let number = "";
        var n = 16;
        for (var i = 0; i < n; i++) {
            number += Math.floor(Math.random() * 10);
        }
        return number;
    };

    function entry_ua(ua) {
        return btoa(ua)
    }
    function getRandomString(length) {
        const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
        let result = '';
        for (let i = 0; i < length; i++) {
            const randomIndex = Math.floor(Math.random() * chars.length);
            result += chars[randomIndex];
        }
        return result;
    }

    function AES_Encrypt(ua, key) {
        var iv = getRandomString(16);
        // var word = `ts=${new Date().getTime()}&fp=628526e50d56b11eaa251aeb40bb5550&ua=${btoa(ua)}&unknown=${btoa('cpuClass,areColorsInverted,osCpu')}&botd=false&hr=0&lhr=31&sr=0`;
        var word = `ts=${new Date().getTime()}&fp=628526e50d56b11eaa251aeb40bb5550&device=PC&engine=Blink&version=131.0.0.0&browser=Chrome&os=Windows 10&botd=-1&kd=0&mc=0&mv=0`;
        var srcs = CryptoJS.enc.Utf8.parse(word);
        var key_ = CryptoJS.enc.Utf8.parse(key.substring(0, 32));
        var encrypted = CryptoJS.AES.encrypt(srcs, key_, {
            iv: CryptoJS.enc.Utf8.parse(iv),
            mode: CryptoJS.mode.CBC,
            padding: CryptoJS.pad.Pkcs7
        });

        var ret = new Uint8Array(atob(encrypted.toString()).split("").map(function (c) {
            return c.charCodeAt(0);
        }));
        var iv_ret = Buffer.from(iv)
        var res = new Uint8Array(iv_ret['length'] + ret['length']);
        res['set'](iv_ret, 0);
        res['set'](ret, iv_ret['length']);
        return btoa(utf8_(res));
    };


    console.log(AES_Encrypt(
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36",
        "b9680fecc269b3633f1c17b9f40b7dbc",
    ))
    """
    ctx = execjs.compile(ds_original_js_code)

    def run(self, ua, first_cookie):
        res = self.ctx.call("AES_Encrypt", ua, first_cookie)
        return {"MM_mq4qQammP3BA3": first_cookie, "MM_1xL8nvQydGCE3": res}


class CreaditGradeASpiderMiddleware:
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, or item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Request or item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)


class CreaditGradeADownloaderMiddleware(Rs6RetryMiddleware, CompanyIpByApiMiddleware):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.
    def __init__(self, settings):
        CompanyIpByApiMiddleware.__init__(self, settings)
        Rs6RetryMiddleware.__init__(self, settings)
        self.session = requests.Session()

    @retry(stop_max_attempt_number=5)
    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        root_url, params = parse_url_params(unquote_url(request.body.decode()))

        src_headers = request.headers.to_unicode_dict()
        src_proxies = request.meta.get("proxies_info")

        request.headers.update({"User-Agent": Faker().user_agent()})
        CompanyIpByApiMiddleware.process_request(self, request, spider)
        headers = request.headers.to_unicode_dict()
        proxies = {
            "https": request.meta.get("proxies_info"),
            "http": request.meta.get("proxies_info"),
        } if request.meta.get("proxies_info") else {}

        if spider.name == 'shanghai_grade':
            self.session.cookies.clear_session_cookies()
            if 'login_wz_yzm' in params:
                url = "https://shanghai.chinatax.gov.cn/newxbwz/servlet/GetshowimgSmall"
                img = self.session.get(url, timeout=5, proxies=proxies, headers=headers).content
                vcode = ddddocr.DdddOcr(ocr=True, show_ad=False).classification(img)
                params.update({"login_wz_yzm": vcode})
                request._set_body('&'.join(f"{k}={v}" for k, v in params.items()).encode("utf-8"))
                cookies = self.session.cookies.get_dict()
                request.headers.update({"Cookie": '; '.join([f"{k}={v}" for k, v in cookies.items() if v])})
                request.cookies.update(cookies)

        elif spider.name == 'xinjiang_grade':
            if 'yzm' in request.body.decode():
                for _ in range(10):
                    url = f'https://etax.xinjiang.chinatax.gov.cn/yhs-web/api/yhsyzm/get?{time.time()}'
                    img = self.session.get(url, timeout=5, proxies=proxies, headers=headers, verify=False).content
                    vcode = ddddocr.DdddOcr(ocr=True, show_ad=False).classification(img)
                    res = self.session.post(
                        f"https://etax.xinjiang.chinatax.gov.cn/yhs-web/api/yhsyzm/check?{time.time()}",
                        json={"checknum": vcode},
                        timeout=5,
                        proxies=proxies,
                        headers=headers
                    )
                    if res.json().get("success"):
                        params = json.loads(request.body.decode())
                        params.update({"yzm": vcode})
                        request._set_body(json.dumps(params).encode("utf-8"))
                        cookies = self.session.cookies.get_dict()
                        request.headers.update({"Cookie": '; '.join([f"{k}={v}" for k, v in cookies.items() if v])})
                        request.cookies.update(cookies)
                        break

        elif spider.name == 'ningbo_grade':
            if 'yzm' in request.body.decode():
                for _ in range(10):
                    url = f'https://etax.ningbo.chinatax.gov.cn/yhs-web/api/yhsyzm/get?{time.time()}'
                    img = self.session.get(url, timeout=5, proxies=proxies, headers=headers, verify=False).content
                    vcode = ddddocr.DdddOcr(ocr=True, show_ad=False).classification(img)
                    res = self.session.post(
                        f"https://etax.ningbo.chinatax.gov.cn/yhs-web/api/yhsyzm/check?{time.time()}",
                        json={"checknum": vcode},
                        timeout=5,
                        proxies=proxies,
                        headers=headers
                    )
                    if res.json().get("success"):
                        params = json.loads(request.body.decode())
                        params.update({"yzm": vcode})
                        request._set_body(json.dumps(params).encode("utf-8"))
                        cookies = self.session.cookies.get_dict()
                        request.headers.update({"Cookie": '; '.join([f"{k}={v}" for k, v in cookies.items() if v])})
                        request.cookies.update(cookies)
                        break

        elif spider.name == 'hebei_grade':
            if 'yzm' in request.body.decode():
                for _ in range(10):
                    url = f'https://etax.hebei.chinatax.gov.cn/yhs-web/api/yhsyzm/get?{time.time()}'
                    img = self._request(url=url, method="GET", timeout=5, proxies=proxies, headers=headers, verify=False).content
                    vcode = ddddocr.DdddOcr(ocr=True, show_ad=False).classification(img)
                    res = self._request(
                        url=f"https://etax.hebei.chinatax.gov.cn/yhs-web/api/yhsyzm/check?{time.time()}",
                        method="POST",
                        json={"checknum": vcode},
                        timeout=5,
                        proxies=proxies,
                        headers=headers
                    )
                    if res.json().get("success"):
                        params = json.loads(request.body.decode())
                        params.update({"yzm": vcode})
                        request._set_body(json.dumps(params).encode("utf-8"))
                        cookies = self.session.cookies.get_dict()
                        request.headers.update({"Cookie": '; '.join([f"{k}={v}" for k, v in cookies.items() if v])})
                        request.cookies.update(cookies)
                        break

        elif spider.name == 'shanxi_grade':
            for i in range(20):
                CompanyIpByApiMiddleware.process_request(self, request, spider)
                proxies = {
                    "https": request.meta.get("proxies_info"),
                    "http": request.meta.get("proxies_info"),
                } if request.meta.get("proxies_info") else {}
                self.session.cookies.clear_session_cookies()
                url = "http://shanxi.chinatax.gov.cn/nsfw/ajnsr/sx-11400"
                response = self._request(url=url, method='GET', headers=headers, proxies=proxies, timeout=5)
                cookies = {**self.session.cookies.get_dict(), **response.cookies.get_dict()}
                request.headers.update({"Cookie": '; '.join([f"{k}={v}" for k, v in cookies.items() if v])})
                request.cookies.update(cookies)
                if response.status_code == 200:
                    break

        elif spider.name == 'shenzhen_grade':
            self.session.cookies.clear_session_cookies()
            url = "https://shenzhen.chinatax.gov.cn/sztaxapp/NsxySearch/index"
            response = self._request(url=url, method='GET', headers=headers, proxies=proxies, timeout=5)
            token = parsel.Selector(response.text).xpath('//input[@name="token"]/@value').get()
            cookies = {**response.cookies.get_dict(), **self.session.cookies.get_dict()}
            request.headers.update({"Cookie": '; '.join([f"{k}={v}" for k, v in cookies.items() if v])})
            request.cookies.update(cookies)
            if 'token' in params:
                params.update({"token": token})
                request._set_body('&'.join(f"{k}={v}" for k, v in params.items()).encode("utf-8"))
        elif spider.name == 'liaoning_grade':
            if "LMCL" in request.url:
                request.headers.update(src_headers)
                request.meta["proxy"] = src_proxies
                request.meta["http_proxy"] = src_proxies
                request.meta["proxies_info"] = src_proxies  # 代理原始信息

        elif spider.name == 'hunan_grade':
            self.session.cookies.clear_session_cookies()
            url = "https://hunan.chinatax.gov.cn/index"
            response = self.session.request(url=url, method='GET', headers=headers, proxies=proxies, timeout=5)
            cookies = {}
            for _ in range(5):
                if response.status_code in [412]:
                    first_cookie = (cookies or response.cookies or self.session.cookies.get_dict()).get("MM_mq4qQammP3BA3")
                    cookies = {**self.session.cookies.get_dict(), **response.cookies.get_dict()}
                    cookies.update(HunanEncrypt().run(str(headers['User-Agent']), first_cookie))
                    self.session.cookies.update(cookies)
                    response = self.session.request(url=url, method='GET', headers=headers, proxies=proxies, timeout=5)
                    cookies = {**self.session.cookies.get_dict(), **response.cookies.get_dict()}
                else:
                    break
            request.headers.update({"Cookie": '; '.join([f"{k}={v}" for k, v in cookies.items() if v])})
            request.cookies.update(cookies)
            csrf = re.findall('"_csrf" content="(.*?)"', response.text)
            if '_csrf' in params and csrf:
                params.update({"_csrf": csrf[0]})
                request._set_body('&'.join(f"{k}={v}" for k, v in params.items()).encode("utf-8"))

        elif spider.name == 'anhui_grade':
            self.session.cookies.clear_session_cookies()
            url = "https://etax.anhui.chinatax.gov.cn/nsr/captcha/numberChineseValidate"
            num_dict = {"壹": "1", "贰": "2", "叁": "3", "肆": "4", "伍": "5", "陆": "6", "柒": "7", "捌": "8", "玖": "9"}
            op_dict = {"加": "+", "减": "-", "乘": "*", "除": "/"}
            for _ in range(5):
                response = self.session.request(url=url, method='GET', headers=headers, proxies=proxies, timeout=5)
                resp_v = ddddocr.DdddOcr(show_ad=False).classification(response.content)
                try:
                    vcode = eval(num_dict[resp_v[0]] + op_dict[resp_v[1]] + num_dict[resp_v[2]])
                    spider.logger.info(f"验证码结果 {vcode}")
                except Exception:
                    continue
                cookies = {**response.cookies.get_dict(), **self.session.cookies.get_dict()}
                request.headers.update({"Cookie": '; '.join([f"{k}={v}" for k, v in cookies.items() if v])})
                request.cookies.update(cookies)
                if 'vcode' in params and vcode:
                    params.update({"vcode": vcode})
                    request._set_body('&'.join(f"{k}={v}" for k, v in params.items()).encode("utf-8"))
                break

        return None

    @retry(stop_max_attempt_number=5)
    def _request(self, **kwargs):
        allowed_methods_kwargs = [
            "params", "data", "headers", "cookies", "files", "auth", "timeout", "allow_redirects", "proxies", "hooks",
            "stream", "verify", "cert", "json"
        ]
        if (kwargs.get("retry_times") or 0) >= 10:
            print("retry times is too many")
            return None
        request_kwargs = {k: v for k, v in kwargs.items() if k in allowed_methods_kwargs}
        response = self.session.request(kwargs.get('method'), kwargs.get('url'), **request_kwargs)
        if response.status_code in [202, 412]:
            rs_cookie = self.gen_new_cookie(response, base_cookie=self.session.cookies.get_dict(), proxy=self.session.proxies, headers=self.session.headers)
            self.session.cookies.update(rs_cookie)
            raise Exception("rs_cookie")
        return response

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        if spider.name == 'liaoning_grade':
            def get_new_url_prefix(response_):
                js_code = re.findall(r"<script>(.*?)</script>", response_.text)
                if js_code:
                    new_url = execjs.compile(
                        "var url;var window = {open: function(a,b) {url = a;}};function get_url(){return url};" + js_code[0]
                    ).call("get_url")
                    prefix = new_url.split("?")[-1]
                    return prefix

            url_prefix = get_new_url_prefix(response)
            if url_prefix:
                root_url, params = parse_url_params(unquote_url(request.url))
                _params = dict([i.split("=") for i in url_prefix.split("&")])
                _params.update(params)
                request = request.replace(url=root_url + "?" + urlencode(_params))
                return self._retry(request, 'retry', spider)

        elif spider.name == 'hunan_grade':
            if response.status in [412, 404]:
                retry_times = request.meta.get('retry_times', 0)
                self.process_request(request, spider)
                request.meta['retry_times'] = retry_times + 1
                return self._retry(request, 'retry', spider)

        elif spider.name == 'anhui_grade':
            if 'vcode' in request.body.decode("utf-8") and response.json().get("code") == 400:
                retry_times = request.meta.get('retry_times', 0)
                self.process_request(request, spider)
                request.meta['retry_times'] = retry_times + 1
                return self._retry(request, 'retry', spider)

        response = Rs6RetryMiddleware.process_response(self, request, response, spider)
        if response:
            return response
        raise IgnoreRequest("已经重试最大次数 丢弃")

    def process_exception(self, request, exception, spider):
        new_request = CompanyIpByApiMiddleware.process_exception(self, request, exception, spider)
        if new_request:
            self.process_request(new_request, spider)
            return new_request
        else:
            raise IgnoreRequest("已经重试最大次数 丢弃")

    def spider_opened(self, spider):
        spider.logger.info("Spider opened: %s" % spider.name)
